{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 手写识别回顾"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "**说明:**\n",
    "\n",
    "将 `/trainingDigits` 和 `/testDigits` 放在当前目录下。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from numpy import *\n",
    "from time import sleep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadDataSet(fileName):\n",
    "    dataMat = []; labelMat = []\n",
    "    fr = open(fileName)\n",
    "    for line in fr.readlines():\n",
    "        lineArr = line.strip().split('\\t')\n",
    "        dataMat.append([float(lineArr[0]), float(lineArr[1])])\n",
    "        labelMat.append(float(lineArr[2]))\n",
    "    return dataMat,labelMat\n",
    "\n",
    "def selectJrand(i,m):\n",
    "    j=i #we want to select any J not equal to i\n",
    "    while (j==i):\n",
    "        j = int(random.uniform(0,m))\n",
    "    return j\n",
    "\n",
    "def clipAlpha(aj,H,L):\n",
    "    if aj > H: \n",
    "        aj = H\n",
    "    if L > aj:\n",
    "        aj = L\n",
    "    return aj"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义核转换函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def kernelTrans(X, A, kTup): #calc the kernel or transform data to a higher dimensional space\n",
    "    m,n = shape(X)\n",
    "    K = mat(zeros((m,1)))\n",
    "    if kTup[0]=='lin': K = X * A.T   #linear kernel\n",
    "    elif kTup[0]=='rbf':\n",
    "        for j in range(m):\n",
    "            deltaRow = X[j,:] - A\n",
    "            K[j] = deltaRow*deltaRow.T\n",
    "        K = exp(K/(-1*kTup[1]**2)) #divide in NumPy is element-wise not matrix like Matlab\n",
    "    else: raise NameError('Houston We Have a Problem -- \\\n",
    "    That Kernel is not recognized')\n",
    "    return K"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义完整版 Platt SMO 支持函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class optStruct:\n",
    "    def __init__(self,dataMatIn, classLabels, C, toler, kTup):  # Initialize the structure with the parameters \n",
    "        self.X = dataMatIn\n",
    "        self.labelMat = classLabels\n",
    "        self.C = C\n",
    "        self.tol = toler\n",
    "        self.m = shape(dataMatIn)[0]\n",
    "        self.alphas = mat(zeros((self.m,1)))\n",
    "        self.b = 0\n",
    "        self.eCache = mat(zeros((self.m,2))) #first column is valid flag\n",
    "        self.K = mat(zeros((self.m,self.m)))\n",
    "        for i in range(self.m):\n",
    "            self.K[:,i] = kernelTrans(self.X, self.X[i,:], kTup)\n",
    "\n",
    "def calcEk(oS, k):\n",
    "    fXk = float(multiply(oS.alphas,oS.labelMat).T*oS.K[:,k] + oS.b)\n",
    "    Ek = fXk - float(oS.labelMat[k])\n",
    "    return Ek\n",
    "        \n",
    "def selectJ(i, oS, Ei):         #this is the second choice -heurstic, and calcs Ej\n",
    "    maxK = -1; maxDeltaE = 0; Ej = 0\n",
    "    oS.eCache[i] = [1,Ei]  #set valid #choose the alpha that gives the maximum delta E\n",
    "    validEcacheList = nonzero(oS.eCache[:,0].A)[0]\n",
    "    if (len(validEcacheList)) > 1:\n",
    "        for k in validEcacheList:   #loop through valid Ecache values and find the one that maximizes delta E\n",
    "            if k == i: continue #don't calc for i, waste of time\n",
    "            Ek = calcEk(oS, k)\n",
    "            deltaE = abs(Ei - Ek)\n",
    "            if (deltaE > maxDeltaE):\n",
    "                maxK = k; maxDeltaE = deltaE; Ej = Ek\n",
    "        return maxK, Ej\n",
    "    else:   #in this case (first time around) we don't have any valid eCache values\n",
    "        j = selectJrand(i, oS.m)\n",
    "        Ej = calcEk(oS, j)\n",
    "    return j, Ej\n",
    "\n",
    "def updateEk(oS, k):#after any alpha has changed update the new value in the cache\n",
    "    Ek = calcEk(oS, k)\n",
    "    oS.eCache[k] = [1,Ek]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义完整版 Platt SMO 算法优化例程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def innerL(i, oS):\n",
    "    Ei = calcEk(oS, i)\n",
    "    if ((oS.labelMat[i]*Ei < -oS.tol) and (oS.alphas[i] < oS.C)) or ((oS.labelMat[i]*Ei > oS.tol) and (oS.alphas[i] > 0)):\n",
    "        j,Ej = selectJ(i, oS, Ei) #this has been changed from selectJrand\n",
    "        alphaIold = oS.alphas[i].copy(); alphaJold = oS.alphas[j].copy();\n",
    "        if (oS.labelMat[i] != oS.labelMat[j]):\n",
    "            L = max(0, oS.alphas[j] - oS.alphas[i])\n",
    "            H = min(oS.C, oS.C + oS.alphas[j] - oS.alphas[i])\n",
    "        else:\n",
    "            L = max(0, oS.alphas[j] + oS.alphas[i] - oS.C)\n",
    "            H = min(oS.C, oS.alphas[j] + oS.alphas[i])\n",
    "        if L==H: print(\"L==H\"); return 0\n",
    "        eta = 2.0 * oS.K[i,j] - oS.K[i,i] - oS.K[j,j] #changed for kernel\n",
    "        if eta >= 0: print(\"eta>=0\"); return 0\n",
    "        oS.alphas[j] -= oS.labelMat[j]*(Ei - Ej)/eta\n",
    "        oS.alphas[j] = clipAlpha(oS.alphas[j],H,L)\n",
    "        updateEk(oS, j) #added this for the Ecache\n",
    "        if (abs(oS.alphas[j] - alphaJold) < 0.00001): print(\"j not moving enough\"); return 0\n",
    "        oS.alphas[i] += oS.labelMat[j]*oS.labelMat[i]*(alphaJold - oS.alphas[j])#update i by the same amount as j\n",
    "        updateEk(oS, i) #added this for the Ecache                    #the update is in the oppostie direction\n",
    "        b1 = oS.b - Ei- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,i] - oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[i,j]\n",
    "        b2 = oS.b - Ej- oS.labelMat[i]*(oS.alphas[i]-alphaIold)*oS.K[i,j]- oS.labelMat[j]*(oS.alphas[j]-alphaJold)*oS.K[j,j]\n",
    "        if (0 < oS.alphas[i]) and (oS.C > oS.alphas[i]): oS.b = b1\n",
    "        elif (0 < oS.alphas[j]) and (oS.C > oS.alphas[j]): oS.b = b2\n",
    "        else: oS.b = (b1 + b2)/2.0\n",
    "        return 1\n",
    "    else: return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义完整版 Platt SMO 的外循环代码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def smoP(dataMatIn, classLabels, C, toler, maxIter,kTup=('lin', 0)):    #full Platt SMO\n",
    "    oS = optStruct(mat(dataMatIn),mat(classLabels).transpose(),C,toler, kTup)\n",
    "    iter = 0\n",
    "    entireSet = True; alphaPairsChanged = 0\n",
    "    while (iter < maxIter) and ((alphaPairsChanged > 0) or (entireSet)):\n",
    "        alphaPairsChanged = 0\n",
    "        if entireSet:   #go over all\n",
    "            for i in range(oS.m):        \n",
    "                alphaPairsChanged += innerL(i,oS)\n",
    "                print(\"fullSet, iter: %d i:%d, pairs changed %d\" % (iter,i,alphaPairsChanged))\n",
    "            iter += 1\n",
    "        else:#go over non-bound (railed) alphas\n",
    "            nonBoundIs = nonzero((oS.alphas.A > 0) * (oS.alphas.A < C))[0]\n",
    "            for i in nonBoundIs:\n",
    "                alphaPairsChanged += innerL(i,oS)\n",
    "                print(\"non-bound, iter: %d i:%d, pairs changed %d\" % (iter,i,alphaPairsChanged))\n",
    "            iter += 1\n",
    "        if entireSet: entireSet = False #toggle entire set loop\n",
    "        elif (alphaPairsChanged == 0): entireSet = True  \n",
    "        print(\"iteration number: %d\" % iter)\n",
    "    return oS.b,oS.alphas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 利用核函数进行分类的径向基测试函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testRbf(k1=1.3):\n",
    "    dataArr,labelArr = loadDataSet('testSetRBF.txt')\n",
    "    b,alphas = smoP(dataArr, labelArr, 200, 0.0001, 10000, ('rbf', k1)) #C=200 important\n",
    "    datMat=mat(dataArr); labelMat = mat(labelArr).transpose()\n",
    "    svInd=nonzero(alphas.A>0)[0]\n",
    "    sVs=datMat[svInd] #get matrix of only support vectors\n",
    "    labelSV = labelMat[svInd];\n",
    "    print(\"there are %d Support Vectors\" % shape(sVs)[0])\n",
    "    m,n = shape(datMat)\n",
    "    errorCount = 0\n",
    "    for i in range(m):\n",
    "        kernelEval = kernelTrans(sVs,datMat[i,:],('rbf', k1))\n",
    "        predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + b\n",
    "        if sign(predict)!=sign(labelArr[i]): errorCount += 1\n",
    "    print(\"the training error rate is: %f\" % (float(errorCount)/m))\n",
    "    dataArr,labelArr = loadDataSet('testSetRBF2.txt')\n",
    "    errorCount = 0\n",
    "    datMat=mat(dataArr); labelMat = mat(labelArr).transpose()\n",
    "    m,n = shape(datMat)\n",
    "    for i in range(m):\n",
    "        kernelEval = kernelTrans(sVs,datMat[i,:],('rbf', k1))\n",
    "        predict=kernelEval.T * multiply(labelSV,alphas[svInd]) + b\n",
    "        if sign(predict)!=sign(labelArr[i]): errorCount += 1    \n",
    "    print(\"the test error rate is: %f\" % (float(errorCount)/m))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fullSet, iter: 0 i:0, pairs changed 1\n",
      "fullSet, iter: 0 i:1, pairs changed 2\n",
      "fullSet, iter: 0 i:2, pairs changed 3\n",
      "fullSet, iter: 0 i:3, pairs changed 4\n",
      "fullSet, iter: 0 i:4, pairs changed 5\n",
      "fullSet, iter: 0 i:5, pairs changed 6\n",
      "fullSet, iter: 0 i:6, pairs changed 6\n",
      "fullSet, iter: 0 i:7, pairs changed 7\n",
      "fullSet, iter: 0 i:8, pairs changed 8\n",
      "fullSet, iter: 0 i:9, pairs changed 8\n",
      "L==H\n",
      "fullSet, iter: 0 i:10, pairs changed 8\n",
      "fullSet, iter: 0 i:11, pairs changed 9\n",
      "fullSet, iter: 0 i:12, pairs changed 9\n",
      "fullSet, iter: 0 i:13, pairs changed 10\n",
      "L==H\n",
      "fullSet, iter: 0 i:14, pairs changed 10\n",
      "fullSet, iter: 0 i:15, pairs changed 11\n",
      "fullSet, iter: 0 i:16, pairs changed 12\n",
      "L==H\n",
      "fullSet, iter: 0 i:17, pairs changed 12\n",
      "fullSet, iter: 0 i:18, pairs changed 13\n",
      "fullSet, iter: 0 i:19, pairs changed 14\n",
      "fullSet, iter: 0 i:20, pairs changed 14\n",
      "L==H\n",
      "fullSet, iter: 0 i:21, pairs changed 14\n",
      "fullSet, iter: 0 i:22, pairs changed 14\n",
      "fullSet, iter: 0 i:23, pairs changed 15\n",
      "fullSet, iter: 0 i:24, pairs changed 16\n",
      "fullSet, iter: 0 i:25, pairs changed 16\n",
      "fullSet, iter: 0 i:26, pairs changed 17\n",
      "fullSet, iter: 0 i:27, pairs changed 18\n",
      "fullSet, iter: 0 i:28, pairs changed 19\n",
      "fullSet, iter: 0 i:29, pairs changed 20\n",
      "fullSet, iter: 0 i:30, pairs changed 20\n",
      "fullSet, iter: 0 i:31, pairs changed 21\n",
      "fullSet, iter: 0 i:32, pairs changed 21\n",
      "fullSet, iter: 0 i:33, pairs changed 21\n",
      "fullSet, iter: 0 i:34, pairs changed 21\n",
      "fullSet, iter: 0 i:35, pairs changed 21\n",
      "L==H\n",
      "fullSet, iter: 0 i:36, pairs changed 21\n",
      "fullSet, iter: 0 i:37, pairs changed 21\n",
      "L==H\n",
      "fullSet, iter: 0 i:38, pairs changed 21\n",
      "fullSet, iter: 0 i:39, pairs changed 21\n",
      "fullSet, iter: 0 i:40, pairs changed 21\n",
      "L==H\n",
      "fullSet, iter: 0 i:41, pairs changed 21\n",
      "L==H\n",
      "fullSet, iter: 0 i:42, pairs changed 21\n",
      "L==H\n",
      "fullSet, iter: 0 i:43, pairs changed 21\n",
      "fullSet, iter: 0 i:44, pairs changed 21\n",
      "fullSet, iter: 0 i:45, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:46, pairs changed 22\n",
      "fullSet, iter: 0 i:47, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:48, pairs changed 22\n",
      "fullSet, iter: 0 i:49, pairs changed 22\n",
      "j not moving enough\n",
      "fullSet, iter: 0 i:50, pairs changed 22\n",
      "fullSet, iter: 0 i:51, pairs changed 22\n",
      "fullSet, iter: 0 i:52, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:53, pairs changed 22\n",
      "fullSet, iter: 0 i:54, pairs changed 22\n",
      "fullSet, iter: 0 i:55, pairs changed 22\n",
      "j not moving enough\n",
      "fullSet, iter: 0 i:56, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:57, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:58, pairs changed 22\n",
      "fullSet, iter: 0 i:59, pairs changed 22\n",
      "fullSet, iter: 0 i:60, pairs changed 22\n",
      "j not moving enough\n",
      "fullSet, iter: 0 i:61, pairs changed 22\n",
      "fullSet, iter: 0 i:62, pairs changed 22\n",
      "fullSet, iter: 0 i:63, pairs changed 22\n",
      "fullSet, iter: 0 i:64, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:65, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:66, pairs changed 22\n",
      "fullSet, iter: 0 i:67, pairs changed 22\n",
      "fullSet, iter: 0 i:68, pairs changed 22\n",
      "fullSet, iter: 0 i:69, pairs changed 22\n",
      "fullSet, iter: 0 i:70, pairs changed 22\n",
      "fullSet, iter: 0 i:71, pairs changed 22\n",
      "fullSet, iter: 0 i:72, pairs changed 22\n",
      "fullSet, iter: 0 i:73, pairs changed 22\n",
      "j not moving enough\n",
      "fullSet, iter: 0 i:74, pairs changed 22\n",
      "fullSet, iter: 0 i:75, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:76, pairs changed 22\n",
      "fullSet, iter: 0 i:77, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:78, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:79, pairs changed 22\n",
      "fullSet, iter: 0 i:80, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:81, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:82, pairs changed 22\n",
      "fullSet, iter: 0 i:83, pairs changed 22\n",
      "fullSet, iter: 0 i:84, pairs changed 22\n",
      "fullSet, iter: 0 i:85, pairs changed 22\n",
      "fullSet, iter: 0 i:86, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:87, pairs changed 22\n",
      "fullSet, iter: 0 i:88, pairs changed 22\n",
      "fullSet, iter: 0 i:89, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:90, pairs changed 22\n",
      "fullSet, iter: 0 i:91, pairs changed 22\n",
      "L==H\n",
      "fullSet, iter: 0 i:92, pairs changed 22\n",
      "fullSet, iter: 0 i:93, pairs changed 22\n",
      "fullSet, iter: 0 i:94, pairs changed 22\n",
      "fullSet, iter: 0 i:95, pairs changed 22\n",
      "fullSet, iter: 0 i:96, pairs changed 22\n",
      "fullSet, iter: 0 i:97, pairs changed 22\n",
      "fullSet, iter: 0 i:98, pairs changed 22\n",
      "fullSet, iter: 0 i:99, pairs changed 22\n",
      "iteration number: 1\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:0, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:1, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:2, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:3, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:7, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:8, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:11, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:13, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:14, pairs changed 0\n",
      "non-bound, iter: 1 i:15, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:16, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:17, pairs changed 1\n",
      "non-bound, iter: 1 i:18, pairs changed 2\n",
      "non-bound, iter: 1 i:19, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:21, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:23, pairs changed 3\n",
      "non-bound, iter: 1 i:24, pairs changed 4\n",
      "non-bound, iter: 1 i:26, pairs changed 5\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:27, pairs changed 5\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:28, pairs changed 5\n",
      "non-bound, iter: 1 i:29, pairs changed 6\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:31, pairs changed 6\n",
      "j not moving enough\n",
      "non-bound, iter: 1 i:45, pairs changed 6\n",
      "iteration number: 2\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:0, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:1, pairs changed 0\n",
      "non-bound, iter: 2 i:2, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:3, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:7, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:8, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:11, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:13, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:14, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:16, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:17, pairs changed 1\n",
      "non-bound, iter: 2 i:18, pairs changed 2\n",
      "non-bound, iter: 2 i:19, pairs changed 3\n",
      "non-bound, iter: 2 i:21, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:23, pairs changed 3\n",
      "non-bound, iter: 2 i:26, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:27, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:28, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:29, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:31, pairs changed 4\n",
      "non-bound, iter: 2 i:45, pairs changed 5\n",
      "non-bound, iter: 2 i:48, pairs changed 6\n",
      "non-bound, iter: 2 i:76, pairs changed 6\n",
      "j not moving enough\n",
      "non-bound, iter: 2 i:87, pairs changed 6\n",
      "iteration number: 3\n",
      "non-bound, iter: 3 i:0, pairs changed 1\n",
      "non-bound, iter: 3 i:1, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:3, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:7, pairs changed 1\n",
      "non-bound, iter: 3 i:8, pairs changed 2\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:11, pairs changed 2\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:13, pairs changed 2\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:14, pairs changed 2\n",
      "non-bound, iter: 3 i:16, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:17, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:18, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:19, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:21, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:23, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:26, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:27, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:28, pairs changed 3\n",
      "non-bound, iter: 3 i:29, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:31, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:36, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:41, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:45, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 3 i:76, pairs changed 4\n",
      "non-bound, iter: 3 i:87, pairs changed 4\n",
      "iteration number: 4\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:0, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:1, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:3, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:7, pairs changed 0\n",
      "non-bound, iter: 4 i:11, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:13, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:14, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:16, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:17, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:18, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:19, pairs changed 1\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:21, pairs changed 1\n",
      "non-bound, iter: 4 i:23, pairs changed 2\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:26, pairs changed 2\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:27, pairs changed 2\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:28, pairs changed 2\n",
      "non-bound, iter: 4 i:29, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:31, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:36, pairs changed 3\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:41, pairs changed 3\n",
      "non-bound, iter: 4 i:45, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:76, pairs changed 4\n",
      "j not moving enough\n",
      "non-bound, iter: 4 i:87, pairs changed 4\n",
      "iteration number: 5\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:0, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:1, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:3, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:7, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:11, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:13, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:14, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:16, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:17, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:18, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:19, pairs changed 0\n",
      "non-bound, iter: 5 i:21, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:23, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:26, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:27, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:28, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:29, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:31, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:36, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:41, pairs changed 0\n",
      "non-bound, iter: 5 i:45, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:76, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 5 i:87, pairs changed 0\n",
      "iteration number: 6\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:0, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:1, pairs changed 0\n",
      "fullSet, iter: 6 i:2, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:3, pairs changed 0\n",
      "fullSet, iter: 6 i:4, pairs changed 0\n",
      "fullSet, iter: 6 i:5, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 6 i:6, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:7, pairs changed 0\n",
      "fullSet, iter: 6 i:8, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:9, pairs changed 0\n",
      "fullSet, iter: 6 i:10, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:11, pairs changed 0\n",
      "fullSet, iter: 6 i:12, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:13, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:14, pairs changed 0\n",
      "fullSet, iter: 6 i:15, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:16, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:17, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:18, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:19, pairs changed 0\n",
      "fullSet, iter: 6 i:20, pairs changed 0\n",
      "fullSet, iter: 6 i:21, pairs changed 0\n",
      "fullSet, iter: 6 i:22, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:23, pairs changed 0\n",
      "fullSet, iter: 6 i:24, pairs changed 0\n",
      "fullSet, iter: 6 i:25, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:26, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:27, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:28, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:29, pairs changed 0\n",
      "fullSet, iter: 6 i:30, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:31, pairs changed 0\n",
      "fullSet, iter: 6 i:32, pairs changed 0\n",
      "fullSet, iter: 6 i:33, pairs changed 0\n",
      "fullSet, iter: 6 i:34, pairs changed 1\n",
      "fullSet, iter: 6 i:35, pairs changed 1\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:36, pairs changed 1\n",
      "fullSet, iter: 6 i:37, pairs changed 1\n",
      "fullSet, iter: 6 i:38, pairs changed 1\n",
      "fullSet, iter: 6 i:39, pairs changed 1\n",
      "fullSet, iter: 6 i:40, pairs changed 1\n",
      "fullSet, iter: 6 i:41, pairs changed 1\n",
      "L==H\n",
      "fullSet, iter: 6 i:42, pairs changed 1\n",
      "fullSet, iter: 6 i:43, pairs changed 1\n",
      "fullSet, iter: 6 i:44, pairs changed 1\n",
      "fullSet, iter: 6 i:45, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:46, pairs changed 2\n",
      "fullSet, iter: 6 i:47, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:48, pairs changed 2\n",
      "fullSet, iter: 6 i:49, pairs changed 2\n",
      "fullSet, iter: 6 i:50, pairs changed 2\n",
      "fullSet, iter: 6 i:51, pairs changed 2\n",
      "fullSet, iter: 6 i:52, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:53, pairs changed 2\n",
      "fullSet, iter: 6 i:54, pairs changed 2\n",
      "fullSet, iter: 6 i:55, pairs changed 2\n",
      "fullSet, iter: 6 i:56, pairs changed 2\n",
      "fullSet, iter: 6 i:57, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:58, pairs changed 2\n",
      "fullSet, iter: 6 i:59, pairs changed 2\n",
      "fullSet, iter: 6 i:60, pairs changed 2\n",
      "fullSet, iter: 6 i:61, pairs changed 2\n",
      "fullSet, iter: 6 i:62, pairs changed 2\n",
      "fullSet, iter: 6 i:63, pairs changed 2\n",
      "fullSet, iter: 6 i:64, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:65, pairs changed 2\n",
      "fullSet, iter: 6 i:66, pairs changed 2\n",
      "fullSet, iter: 6 i:67, pairs changed 2\n",
      "fullSet, iter: 6 i:68, pairs changed 2\n",
      "fullSet, iter: 6 i:69, pairs changed 2\n",
      "fullSet, iter: 6 i:70, pairs changed 2\n",
      "fullSet, iter: 6 i:71, pairs changed 2\n",
      "fullSet, iter: 6 i:72, pairs changed 2\n",
      "fullSet, iter: 6 i:73, pairs changed 2\n",
      "fullSet, iter: 6 i:74, pairs changed 2\n",
      "fullSet, iter: 6 i:75, pairs changed 2\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:76, pairs changed 2\n",
      "fullSet, iter: 6 i:77, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:78, pairs changed 2\n",
      "fullSet, iter: 6 i:79, pairs changed 2\n",
      "fullSet, iter: 6 i:80, pairs changed 2\n",
      "fullSet, iter: 6 i:81, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:82, pairs changed 2\n",
      "fullSet, iter: 6 i:83, pairs changed 2\n",
      "fullSet, iter: 6 i:84, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:85, pairs changed 2\n",
      "fullSet, iter: 6 i:86, pairs changed 2\n",
      "j not moving enough\n",
      "fullSet, iter: 6 i:87, pairs changed 2\n",
      "fullSet, iter: 6 i:88, pairs changed 2\n",
      "fullSet, iter: 6 i:89, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:90, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:91, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:92, pairs changed 2\n",
      "fullSet, iter: 6 i:93, pairs changed 2\n",
      "fullSet, iter: 6 i:94, pairs changed 2\n",
      "fullSet, iter: 6 i:95, pairs changed 2\n",
      "L==H\n",
      "fullSet, iter: 6 i:96, pairs changed 2\n",
      "fullSet, iter: 6 i:97, pairs changed 2\n",
      "fullSet, iter: 6 i:98, pairs changed 2\n",
      "fullSet, iter: 6 i:99, pairs changed 2\n",
      "iteration number: 7\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:0, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:3, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:7, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:11, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:13, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:14, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:16, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:17, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:18, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:19, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:21, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:23, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:26, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:27, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:28, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:29, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:31, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:34, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:36, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:41, pairs changed 0\n",
      "non-bound, iter: 7 i:45, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:76, pairs changed 0\n",
      "j not moving enough\n",
      "non-bound, iter: 7 i:87, pairs changed 0\n",
      "iteration number: 8\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:0, pairs changed 0\n",
      "fullSet, iter: 8 i:1, pairs changed 0\n",
      "fullSet, iter: 8 i:2, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:3, pairs changed 0\n",
      "fullSet, iter: 8 i:4, pairs changed 0\n",
      "fullSet, iter: 8 i:5, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:6, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:7, pairs changed 0\n",
      "fullSet, iter: 8 i:8, pairs changed 0\n",
      "fullSet, iter: 8 i:9, pairs changed 0\n",
      "fullSet, iter: 8 i:10, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:11, pairs changed 0\n",
      "fullSet, iter: 8 i:12, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:13, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:14, pairs changed 0\n",
      "fullSet, iter: 8 i:15, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:16, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:17, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:18, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:19, pairs changed 0\n",
      "fullSet, iter: 8 i:20, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:21, pairs changed 0\n",
      "fullSet, iter: 8 i:22, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:23, pairs changed 0\n",
      "fullSet, iter: 8 i:24, pairs changed 0\n",
      "fullSet, iter: 8 i:25, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:26, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:27, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:28, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:29, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:30, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:31, pairs changed 0\n",
      "fullSet, iter: 8 i:32, pairs changed 0\n",
      "fullSet, iter: 8 i:33, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:34, pairs changed 0\n",
      "fullSet, iter: 8 i:35, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:36, pairs changed 0\n",
      "fullSet, iter: 8 i:37, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:38, pairs changed 0\n",
      "fullSet, iter: 8 i:39, pairs changed 0\n",
      "fullSet, iter: 8 i:40, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:41, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:42, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:43, pairs changed 0\n",
      "fullSet, iter: 8 i:44, pairs changed 0\n",
      "fullSet, iter: 8 i:45, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:46, pairs changed 0\n",
      "fullSet, iter: 8 i:47, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:48, pairs changed 0\n",
      "fullSet, iter: 8 i:49, pairs changed 0\n",
      "fullSet, iter: 8 i:50, pairs changed 0\n",
      "fullSet, iter: 8 i:51, pairs changed 0\n",
      "fullSet, iter: 8 i:52, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:53, pairs changed 0\n",
      "fullSet, iter: 8 i:54, pairs changed 0\n",
      "fullSet, iter: 8 i:55, pairs changed 0\n",
      "fullSet, iter: 8 i:56, pairs changed 0\n",
      "fullSet, iter: 8 i:57, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:58, pairs changed 0\n",
      "fullSet, iter: 8 i:59, pairs changed 0\n",
      "fullSet, iter: 8 i:60, pairs changed 0\n",
      "fullSet, iter: 8 i:61, pairs changed 0\n",
      "fullSet, iter: 8 i:62, pairs changed 0\n",
      "fullSet, iter: 8 i:63, pairs changed 0\n",
      "fullSet, iter: 8 i:64, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:65, pairs changed 0\n",
      "fullSet, iter: 8 i:66, pairs changed 0\n",
      "fullSet, iter: 8 i:67, pairs changed 0\n",
      "fullSet, iter: 8 i:68, pairs changed 0\n",
      "fullSet, iter: 8 i:69, pairs changed 0\n",
      "fullSet, iter: 8 i:70, pairs changed 0\n",
      "fullSet, iter: 8 i:71, pairs changed 0\n",
      "fullSet, iter: 8 i:72, pairs changed 0\n",
      "fullSet, iter: 8 i:73, pairs changed 0\n",
      "fullSet, iter: 8 i:74, pairs changed 0\n",
      "fullSet, iter: 8 i:75, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:76, pairs changed 0\n",
      "fullSet, iter: 8 i:77, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:78, pairs changed 0\n",
      "fullSet, iter: 8 i:79, pairs changed 0\n",
      "fullSet, iter: 8 i:80, pairs changed 0\n",
      "fullSet, iter: 8 i:81, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:82, pairs changed 0\n",
      "fullSet, iter: 8 i:83, pairs changed 0\n",
      "fullSet, iter: 8 i:84, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:85, pairs changed 0\n",
      "fullSet, iter: 8 i:86, pairs changed 0\n",
      "j not moving enough\n",
      "fullSet, iter: 8 i:87, pairs changed 0\n",
      "fullSet, iter: 8 i:88, pairs changed 0\n",
      "fullSet, iter: 8 i:89, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:90, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:91, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:92, pairs changed 0\n",
      "fullSet, iter: 8 i:93, pairs changed 0\n",
      "fullSet, iter: 8 i:94, pairs changed 0\n",
      "fullSet, iter: 8 i:95, pairs changed 0\n",
      "L==H\n",
      "fullSet, iter: 8 i:96, pairs changed 0\n",
      "fullSet, iter: 8 i:97, pairs changed 0\n",
      "fullSet, iter: 8 i:98, pairs changed 0\n",
      "fullSet, iter: 8 i:99, pairs changed 0\n",
      "iteration number: 9\n",
      "there are 23 Support Vectors\n",
      "the training error rate is: 0.070000\n",
      "the test error rate is: 0.140000\n"
     ]
    }
   ],
   "source": [
    "testRbf()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
